import requests
from bs4 import BeautifulSoup as bs
import json
from sql_helper import MySqlHelper


def read_jd_list(url, item_dict_all=None):
    if item_dict_all is None:
        item_dict_all = dict()
    response = requests.get(url)
    if response.status_code == 200:
        bs_obj = bs(response.text, 'html.parser')
        li_list = bs_obj.find(id='plist').find_all('li')

        for item in li_list:
            item_dict = dict()
            box = item.find('div')
            sku_id = box.attrs['data-sku']
            link_img = box.find('a')
            detail_url = link_img.attrs['href']
            link_img = link_img.find('img')

            if 'src' in link_img.attrs:
                link_img_url = link_img.attrs['src']
            elif 'data-lazy-img' in link_img.attrs:
                link_img_url = link_img.attrs['data-lazy-img']
            else:
                link_img_url = ""
            name_div = box.select_one('div.p-name')
            title = name_div.find('em').text.strip()
            item_dict['sku_id'] = sku_id
            item_dict['detail_url'] = detail_url
            item_dict['link_img_url'] = link_img_url
            item_dict['title'] = title
            item_dict['price'] = 0
            if sku_id not in item_dict_all:
                item_dict_all[sku_id] = item_dict
    else:
        print("您抓的出问题了！")
    return item_dict_all


def read_item_price(skuid_list, item_dict_all):
    url = "https://p.3.cn/prices/mgets?callback=jQuery1130452&type=1&area=1_72_4137_0&skuIds=%s" \
        "&pdbp=0&pdtk=&pdpin=&pduid=138612123&source=list_pc_front&_=1545307586363"
    skuids = "%2CJ_".join(skuid_list)
    url = url % skuids
    response = requests.get(url)
    price_str = response.text[response.text.find('['): -3]
    sku_price_list = json.loads(price_str)
    for sku_price in sku_price_list:
        sku_id = sku_price['id'][2:]
        if sku_id in item_dict_all:
            item_dict_all[sku_id]['price'] = float(sku_price['p'])


if __name__ == "__main__":
    urls = ["https://list.jd.com/index.html?cat=670,729&page={}&sort=sort_rank_asc&trans=1&JL=6_0_0#J_main"
            .format(str(i)) for i in range(1, 11)]
    for url in urls:
        item_dict_all = read_jd_list(url)
        print(item_dict_all.keys())
        skuid_list = [sku for sku in item_dict_all.keys()]
        print(skuid_list)
        read_item_price(skuid_list[:30], item_dict_all)
        read_item_price(skuid_list[30:], item_dict_all)

        mysql = MySqlHelper()
        sql = "INSERT INTO jd_item(sku_id, detail_url, list_img_url, title, price) VALUES(%s,%s,%s,%s,%s)"
        values_list = [list(value.values()) for value in item_dict_all.values()]
        mysql.exec_many(sql, values_list)